<!DOCTYPE html>
<html  lang="zh-CN" >
    <head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, minimum-scale=1, initial-scale=1, maximum-scale=5, viewport-fit=cover">
    <title>记一次线上问题（监控的重要性） | 在路上</title>
    <meta name="description" content="记一次线上问题（监控的重要性）">
<meta property="og:type" content="article">
<meta property="og:title" content="记一次线上问题（监控的重要性）">
<meta property="og:url" content="http://www.lushunming.com.cn/2019/09/13/web/2019-09-13-%E8%AE%B0%E4%B8%80%E6%AC%A1%E7%BA%BF%E4%B8%8A%E9%97%AE%E9%A2%98/index.html">
<meta property="og:site_name" content="在路上">
<meta property="og:description" content="记一次线上问题（监控的重要性）">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2019-09-12T16:00:00.000Z">
<meta property="article:modified_time" content="2019-11-24T07:17:53.644Z">
<meta property="article:author" content="machine">
<meta property="article:tag" content="sql">
<meta property="article:tag" content="性能监控">
<meta property="article:tag" content="javamelody">
<meta name="twitter:card" content="summary">

    
    <link rel="icon" href="/images/favicon.ico" type="image/x-icon">

    
<link rel="stylesheet" href="/css/common.min.css">



    
    
    
        <link href="//cdn.jsdelivr.net/npm/gitalk@1.4.0/dist/gitalk.min.css" rel="stylesheet">
    
    
        <link href="//cdn.jsdelivr.net/npm/lightgallery.js@1.1.3/dist/css/lightgallery.min.css" rel="stylesheet">
    
    
    
<link rel="stylesheet" href="/css/iconfont.min.css">

    
<meta name="generator" content="Hexo 5.1.1"></head>

    <body>
        <header class="header header-fixture">
    <div class="profile-search-wrap flex sm:block">
        
        
        <div class="profile sm:text-center md:px-1 lg:px-3 sm:pb-4 sm:pt-6">
            <a id="avatar" role="link" href="https://www.lushunming.com.cn" class="inline-block lg:w-16 lg:h-16 w-8 h-8 m-2" target="_self" rel="noopener" rel="noreferrer" >
                <img src="/images/avatar.jpg" class="rounded-full" alt="avatar">
            </a>
            <h2 id="name" class="hidden lg:block">lushunming</h2>
            <h3 id="title" class="hidden xl:block">前码农</h3>
            
            <small id="location" class="hidden lg:block">
                <i class="iconfont icon-map-icon"></i>
                泰州
            </small>
            
        </div>
        
        
<div class="search flex-1 flex lg:inline-block sm:hidden lg:px-4 lg:mt-2 lg:mb-4 lg:w-full">
    <form id="search-form" class="my-auto flex-1 lg:border lg:border-solid lg:border-gray-200">
        <div class="input-group table bg-gray-100 lg:bg-white w-full">
            <input id="search-input" type="text" placeholder="搜索" class="inline-block w-full bg-gray-100 lg:bg-white">
            <span class="table-cell">
                <button name="search tigger button" disabled>
                    <i class="iconfont icon-search m-2"></i>
                </button>
            </span>
        </div>
    </form>
        
<script id="search-teamplate" type="text/html" data-path="/content.json">
    <div>
        <div class="search-header bg-gray-400">
            <input id="actual-search-input" model="keyword" ref="input" class="inline-block w-full h-10 px-2 py-1" placeholder="搜索" type="text">
        </div>
        <div class="search-result bg-gray-200">
            {{#each searchPosts}}
            <a href="/{{ path }}" class="result-item block px-2 pb-3 mb-1 pt-1 hover:bg-indigo-100">
                <i class="iconfont icon-file"></i>
                <h1 class="result-title inline font-medium text-lg">{{ title }}</h1>
                <p class="result-content text-gray-600 text-sm">{{{ text }}}</p>
            </a>
            {{/each}}
        </div>
    </div>
</script>

</div>


        <button name="menu toogle button" id="menu-toggle-btn" class="block sm:hidden p-3" role="button" aria-expanded="false">
            <i class="iconfont icon-hamburger"></i>
        </button>
    </div>
    <nav id="menu-nav" class="hidden sm:flex flex-col">
        
        
            <div class="menu-item menu-home" role="menuitem">
                <a href="/.">
                    <i class="iconfont icon-home" aria-hidden="true"></i>
                    <span class="menu-title">首页</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-archives" role="menuitem">
                <a href="/archives">
                    <i class="iconfont icon-archive" aria-hidden="true"></i>
                    <span class="menu-title">归档</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-categories" role="menuitem">
                <a href="/categories">
                    <i class="iconfont icon-folder" aria-hidden="true"></i>
                    <span class="menu-title">分类</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-tags" role="menuitem">
                <a href="/tags">
                    <i class="iconfont icon-tag" aria-hidden="true"></i>
                    <span class="menu-title">标签</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-repository" role="menuitem">
                <a href="/repository">
                    <i class="iconfont icon-project" aria-hidden="true"></i>
                    <span class="menu-title">项目</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-links" role="menuitem">
                <a href="/links">
                    <i class="iconfont icon-friend" aria-hidden="true"></i>
                    <span class="menu-title">友链</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-about" role="menuitem">
                <a href="/about">
                    <i class="iconfont icon-cup" aria-hidden="true"></i>
                    <span class="menu-title">关于</span>
                </a>
            </div>
        
        
<div class="social-links flex sm:flex-col lg:hidden mt-5">
    
        <span class="social-item text-center">
            <a target="_blank" rel="noopener" href="https://github.com/lushunming">
                <i class="iconfont social-icon icon-github"></i>
                <span class="menu-title hidden lg:inline">menu.github</span>
            </a>
        </span>
    
        <span class="social-item text-center">
            <a href="/atom.xml">
                <i class="iconfont social-icon icon-rss"></i>
                <span class="menu-title hidden lg:inline">menu.rss</span>
            </a>
        </span>
    
</div>


    </nav>
</header>

        <section class="main-section">
            
    <main class="flex-1 px-4 py-12 md:px-5 lg:px-8 lg:py-4 relative min-h-screen">
    

    <article class="content article article-archives article-type-list" itemscope="">
        <header class="article-header">
            
    
        <h1 class="article-title text-lg" itemprop="name">
            记一次线上问题（监控的重要性）
        </h1>
    



            <p class="article-meta mb-3 text-xs">
                <span class="article-date">
    <i class="iconfont icon-calendar-check"></i>
	<a href="/2019/09/13/web/2019-09-13-%E8%AE%B0%E4%B8%80%E6%AC%A1%E7%BA%BF%E4%B8%8A%E9%97%AE%E9%A2%98/" class="article-date">
	  <time datetime="2019-09-12T16:00:00.000Z" itemprop="datePublished">9月 13</time>
	</a>
</span>

                
    <span class="article-category">
    <i class="iconfont icon-folder"></i>
    <a class="article-category-link" href="/categories/web/">web</a>
  </span>


                
    <span class="article-tags">
    <i class="iconfont icon-tag"></i>
    <a class="article-tag-none-link" href="/tags/javamelody/" rel="tag">javamelody</a>, <a class="article-tag-none-link" href="/tags/sql/" rel="tag">sql</a>, <a class="article-tag-none-link" href="/tags/%E6%80%A7%E8%83%BD%E7%9B%91%E6%8E%A7/" rel="tag">性能监控</a>
  </span>


                <span class="_partial/post-comment"><i class="icon icon-comment"></i>
                    <a href="/2019/09/13/web/2019-09-13-%E8%AE%B0%E4%B8%80%E6%AC%A1%E7%BA%BF%E4%B8%8A%E9%97%AE%E9%A2%98/#comments" class="article-comment-link">
                        评论
                    </a>
                </span>
                

            </p>
        </header>
        <div class="marked-body article-body">
            <h1>记一次线上问题（监控的重要性）</h1>
<hr>
<h3 id="起因">起因</h3>
<p>一天早上，突然线上的系统变得很慢，大家连登陆都很慢。打开日志，发现很多的<code>java.io.IOException: 远程主机强迫关闭了一个现有的连接</code>和数据库连接timeout。</p>
<h3 id="分析">分析</h3>
<p>看到这些错，大于猜测是数据库连接的问题了。1.连接池出了问题？不可能啊，咱用的是<code>HiKariCP</code>啊，性能最强悍啊。2.难道连接池配置有问题？看了下，完全按照官方公式计算的（<code>connections =（（core_count * 2）+ effective_spindle_count）</code>）。3.难道是并发太大，已经达到了瓶颈？但是cpu的使用率并不是很高。</p>
<h3 id="发现">发现</h3>
<p>在看日志的时候，返现一个页面发文的频率很高。原来是客户今天突然收到指示，集体操作同一个模块。那是不是可能是某个sql执行效率很慢，导致了查询堆积，而且出现了很多timeout？找出sql在正式数据库执行，果然需要好几秒才能返回数据。原来是因为在测试库的数据量远远小于正式库，导致问题没有及时的暴露出来。加上索引做了优化，系统立即恢复了！！！</p>
<h3 id="如何防止">如何防止</h3>
<p>由于数据库不归我们管理，是客户找专门的公司维护，我们的账号没有查看慢查询的权力。那我们能不能在我们自己的系统加入监控功能，定期的查看哪些有问题的sql我们可以及时解决呢？</p>
<ol>
<li>druid：阿里开源的连接池，可以监控，但是我们有最强悍的<code>HiKariCP</code>,待定</li>
<li>能不能在mybatis上想办法，也确实存在插件，可以记录到日志中，但是不利于查找。</li>
<li>javamelody:意外的发现，他可以与Spring集成在一起，可以监控cpu，sql，接口，jsp等，并且还有可视化的图标，一目了然，很符合我们的要求，于是准备在项目中集成他，定期监控情况，对有问题的接口、sql及时优化，而不至于再出现这样的突发状况。</li>
</ol>

        </div>
        
<blockquote class="copyright">
    <p><strong>本文链接 : </strong><a class="permalink" href="http://www.lushunming.com.cn/2019/09/13/web/2019-09-13-%E8%AE%B0%E4%B8%80%E6%AC%A1%E7%BA%BF%E4%B8%8A%E9%97%AE%E9%A2%98/">http://www.lushunming.com.cn/2019/09/13/web/2019-09-13-记一次线上问题/</a></p>
    <p><strong>This article is available under <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank" rel="noopener noreferrer">Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0)</a> License</strong></p>
</blockquote>


    </article>
    
    <section id="comments">
        

        
    </section>


    

</main>


<aside style="" id="sidebar" class="aside aside-fixture">
    <div class="toc-sidebar">
        <nav id="toc" class="article-toc">
            <h3 class="toc-title">文章目录</h3>
            <ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link"><span class="toc-number">1.</span> <span class="toc-text">记一次线上问题（监控的重要性）</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%B5%B7%E5%9B%A0"><span class="toc-number">1.0.1.</span> <span class="toc-text">起因</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%86%E6%9E%90"><span class="toc-number">1.0.2.</span> <span class="toc-text">分析</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%8F%91%E7%8E%B0"><span class="toc-number">1.0.3.</span> <span class="toc-text">发现</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%A6%82%E4%BD%95%E9%98%B2%E6%AD%A2"><span class="toc-number">1.0.4.</span> <span class="toc-text">如何防止</span></a></li></ol></li></ol></li></ol>
        </nav>
    </div>
</aside>





        </section>
        <footer class="hidden lg:block fixed bottom-0 left-0 sm:w-1/12 lg:w-1/6 bg-gray-100 z-40">
    
    <div class="footer-social-links">
        
            <a target="_blank" rel="noopener" href="https://github.com/lushunming">
                <i class="iconfont icon-github"></i>
            </a>
        
            <a href="/atom.xml">
                <i class="iconfont icon-rss"></i>
            </a>
        
    </div>
    
    
</footer>

        <div id="mask" class="hidden mask fixed inset-0 bg-gray-900 opacity-75 z-40"></div>
        <div id="search-view-container" class="hidden shadow-xl"></div>
        
<script src="/js/dom-event.min.js"></script>

<script src="//cdn.jsdelivr.net/npm/yox@1.0.0-alpha.121/dist/standard/prod/yox.min.js"></script>


<script src="/js/search.min.js"></script>


    <script src="//cdn.jsdelivr.net/npm/gitalk@1.5.2/dist/gitalk.min.js"></script>
<script src="//cdn.jsdelivr.net/npm/blueimp-md5@2.10.0/js/md5.min.js"></script>
<script type="text/javascript">
    var gitalk = new Gitalk({
        clientID: 'd76d74420825181b7018',
        clientSecret: '8b9c3676695fa33d6b10d9697a96ef27c8301930',
        repo: 'blog-comment',
        owner: 'lushunming',
        admin: ['lushunming'],
        id: md5(location.pathname),
        distractionFreeMode: true
    })
    gitalk.render('comments')
</script>



    <script src="//cdn.jsdelivr.net/npm/lightgallery.js@1.1.3/dist/js/lightgallery.min.js"></script>
    
<script src="/js/light-gallery.min.js"></script>





    </body>
</html>
